const fs = require('fs');
const cheerio = require('cheerio');
const puppeteer = require('puppeteer-core');

(async () => {
    // 初始化puppeteer
    const browser = await puppeteer.launch({
        executablePath: 'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
        headless: true  //headless默认为true表示不启动GUI
    });

    // 获取page实例
    const page = await browser.newPage();

    // 将webdriver字段删除，防止反爬虫
    await page.evaluateOnNewDocument(()=>{
        const newProto = navigator.__proto__;
        delete newProto.webdriver;
        navigator.__proto__ = newProto;
    })

    // 设置useragent，如果headless设置为true，则必做
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.41');

    url_prefix = 'https://kns.cnki.net'
    
    var file = fs.readFileSync('./5g文献.json')
    var urls = JSON.parse(file)
    // urls的结构为一个list列表
    // 每个元素为num、src、title
    // 完整的链接为 url_prefix + urls.list[i].src

    var content,$
    // 用一个字典保存数据，格式为：{关键词：出现次数}
    var keywords_map = new Map();

    for(let i = 0; i < urls.list.length; i++)
    {
        console.log("正在抓取第",i+1,"页的关键词")
        const url = urls.list[i].src
        await page.goto(url_prefix + url,{waitUntil:'networkidle2'})
        content = await page.content()
        $ = cheerio.load(content)
        const keyword = $('.keywords a')
        keyword.map(function(index,item){
            let aword = $(item).text().trim().replace(';',"")
            if(keywords_map.has(aword))
            {
                let temp = keywords_map.get(aword)
                keywords_map.delete(aword)
                keywords_map.set(aword,++temp)
            }
            else
            {
                keywords_map.set(aword, 1)
            }
        })
    }
    
    // Map不能直接转为Json，需要先转为对象
    let obj = Object.create(null)
    for(let[k,v] of keywords_map){
        obj[k] = v;
    }
    fs.writeFileSync('5g_keyword.json',JSON.stringify(obj))

    await browser.close();
})()